{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "21a4bfd3-062a-4c41-baa7-06afe1d6d11e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 首先是数据集。我的环境中KITTI和VOC是自带的，只需要把它们的格式转化即可。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "014c8394-c576-48f7-bd9d-7b5b165fa4e9",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import cv2\n",
    "from MyTools import kitti_2_yolo\n",
    "import random\n",
    "import shutil\n",
    "import sys"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "83d6748e-2cff-41e3-a2a4-634e3518107d",
   "metadata": {},
   "source": [
    "首先是KITTI数据集。\n",
    "\n",
    "执行这个可以把label转化为YOLOv5格式的。\n",
    "\n",
    "下载完后建议放在img_path的位置。需要把KITTI数据集自带的testing删除（因为他没有label，这里用不上），以免影响处理。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "6199de7a-bba7-49d8-bc59-66e648c494eb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "1000\n",
      "2000\n",
      "3000\n",
      "4000\n",
      "5000\n",
      "6000\n",
      "7000\n"
     ]
    }
   ],
   "source": [
    "label_path = '../training/label_2' #'../test_label'\n",
    "img_path = '../training/image_2' #'../test_img'\n",
    "kitti_2_yolo(img_path, label_path, save_path = '../converted_label/k_2_y')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "897acbdc-816e-4f11-9ed7-61fb4c019b42",
   "metadata": {},
   "source": [
    "执行split_dataset来划分数据集，并且把划分后的数据集放到kitti.yaml中指定的位置\n",
    "\n",
    "这里原始的代码中做的太乱了，我重新整理成了split_dataset，这也导致划分和我的实验不太一样，毕竟是随即划分，可能导致训练的结果有略微不同。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "22df3a24-5e67-423e-a7c0-13aa9b1faea8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "已完成 0% (0/4189)\n",
      "已完成 10% (418/4189)\n",
      "已完成 20% (836/4189)\n",
      "已完成 30% (1254/4189)\n",
      "已完成 40% (1672/4189)\n",
      "已完成 50% (2090/4189)\n",
      "已完成 60% (2508/4189)\n",
      "已完成 70% (2926/4189)\n",
      "已完成 80% (3344/4189)\n",
      "已完成 90% (3762/4189)\n",
      "已完成 100% (4180/4189)\n",
      "已完成 0% (0/4189)\n",
      "已完成 10% (418/4189)\n",
      "已完成 20% (836/4189)\n",
      "已完成 0% (0/4189)\n",
      "已完成 10% (418/4189)\n",
      "已完成 20% (836/4189)\n",
      "已完成 30% (1254/4189)\n",
      "已完成 40% (1672/4189)\n",
      "已完成 50% (2090/4189)\n"
     ]
    }
   ],
   "source": [
    "from Dataset_process import *\n",
    "\n",
    "split_dataset(img_path = '../training/image_2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "82c337e4-02a9-4fa3-b277-4fb09d97288e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "29151b10-0749-4dbc-a1a1-1d72ab227afb",
   "metadata": {},
   "source": [
    "然后来处理VOC数据集。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b51d3398-8319-4f43-8b91-b0058edf550f",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train2012: 100%|██████████| 5717/5717 [00:01<00:00, 3141.67it/s]\n",
      "val2012: 100%|██████████| 5823/5823 [00:01<00:00, 3226.73it/s]\n",
      "train2007: 100%|██████████| 2501/2501 [00:00<00:00, 3376.51it/s]\n",
      "val2007: 100%|██████████| 2510/2510 [00:00<00:00, 3359.40it/s]\n",
      "test2007: 100%|██████████| 4952/4952 [00:01<00:00, 3412.77it/s]\n"
     ]
    }
   ],
   "source": [
    "from Dataset_process import *\n",
    "from utils.general import download, Path\n",
    "\n",
    "root = '../VOCdevkit'\n",
    "VOC2007 = os.path.join(root, 'VOC2007')\n",
    "VOC2012 = os.path.join(root, 'VOC2012')\n",
    "VOC_split(root, VOC2007, VOC2012)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "32b57131-ca21-4dcd-836d-755ec28a3be3",
   "metadata": {},
   "source": [
    "把图片放到VOC.yaml的指定位置。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "9444b703-52cb-4459-a47f-5ca03f40013b",
   "metadata": {},
   "outputs": [],
   "source": [
    "mv_VOC_command = f\"\"\"\n",
    "mkdir ../datasets/VOC && mkdir ../datasets/VOC/images && mkdir ../datasets/VOC/labels/ &&\n",
    "mkdir ../datasets/VOC/images/train && mkdir ../datasets/VOC/images/val && mkdir ../datasets/VOC/images/test && \n",
    "mkdir ../datasets/VOC/labels/train && mkdir ../datasets/VOC/labels/val && mkdir ../datasets/VOC/labels/test && \n",
    "\n",
    "mv {root}/images/train2012 ../datasets/VOC/images &&\n",
    "mv {root}/images/train2007 ../datasets/VOC/images &&\n",
    "mv {root}/images/val2012 ../datasets/VOC/images &&\n",
    "mv {root}/images/val2007 ../datasets/VOC/images &&\n",
    "cp -r {root}/images/test2007 ../datasets/VOC/images &&\n",
    "mv {root}/images/test2007 ../datasets/VOC/images &&\n",
    "\n",
    "mv {root}/labels/train2012 ../datasets/VOC/labels &&\n",
    "mv {root}/labels/train2007 ../datasets/VOC/labels &&\n",
    "mv {root}/labels/val2012 ../datasets/VOC/labels &&\n",
    "mv {root}/labels/val2007 ../datasets/VOC/labels &&\n",
    "cp -r {root}/labels/test2007 ../datasets/VOC/labels &&\n",
    "mv {root}/labels/test2007 ../datasets/VOC/labels\n",
    "\"\"\"\n",
    "!{mv_VOC_command}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "451230f8-6f40-4257-b56f-35bc30561c09",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "abe4e8b4-9e79-4f64-8cf4-1000c5ddf7ad",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "raw",
   "id": "2a564103-73a8-4513-8d0d-193bbe97eefa",
   "metadata": {},
   "source": [
    "然后下载openimages数据集。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ac91f0e3-817e-4485-85cc-f151c9f53b5c",
   "metadata": {},
   "outputs": [],
   "source": [
    "from Dataset_process import *\n",
    "download_openimages()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "69d15166-a30e-40b1-b424-8c2ba6c39bfb",
   "metadata": {},
   "source": [
    "移动到指定位置"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "09f69f24-fd8e-45b8-aa02-63bfef0c8ffc",
   "metadata": {},
   "outputs": [],
   "source": [
    "command = f\"\"\" \\\n",
    "rm -rf ../datasets/openimages/* && \\\n",
    "mkdir ../datasets/openimages && \\\n",
    "mkdir ../datasets/openimages/images && \\\n",
    "mkdir ../datasets/openimages/labels && \\\n",
    "mkdir ../datasets/openimages/images/train/ && \\\n",
    "mkdir ../datasets/openimages/images/val/ && \\\n",
    "mkdir ../datasets/openimages/images/test/ && \\\n",
    "mkdir ../datasets/openimages/labels/train/ && \\\n",
    "mkdir ../datasets/openimages/labels/val/ && \\\n",
    "mkdir ../datasets/openimages/labels/test/ && \\\n",
    "cp ../openimages/train/images/val/* ../datasets/openimages/images/train/ && \\\n",
    "cp ../openimages/val/images/val/* ../datasets/openimages/images/val/ && \\\n",
    "cp ../openimages/test/images/val/* ../datasets/openimages/images/test/ && \\\n",
    "cp ../openimages/train/labels/val/* ../datasets/openimages/labels/train/ && \\\n",
    "cp ../openimages/val/labels/val/* ../datasets/openimages/labels/val/ && \\\n",
    "cp ../openimages/test/labels/val/* ../datasets/openimages/labels/test/ \\\n",
    "\"\"\"\n",
    "!{command}\n",
    "#18：58"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fe01d723-ad9b-4ed2-8bc8-e1802581c371",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "492000da-5dba-4475-8ddb-c961b0fb854c",
   "metadata": {},
   "source": [
    "最后下载VisDrone数据集"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "30c58bae-6c21-4472-bd99-ebdf0808385a",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/autodl-tmp/yolo_incremental_learning/utils/general.py:32: UserWarning: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html. The pkg_resources package is slated for removal as early as 2025-11-30. Refrain from using this package or pin to Setuptools<81.\n",
      "  import pkg_resources as pkg\n",
      "Downloading https://github.com/ultralytics/assets/releases/download/v0.0.0/VisDrone2019-DET-val.zip to /root/autodl-tmp/VisDrone/VisDrone2019-DET-val.zip...\n",
      "Downloading https://github.com/ultralytics/assets/releases/download/v0.0.0/VisDrone2019-DET-test-challenge.zip to /root/autodl-tmp/VisDrone/VisDrone2019-DET-test-challenge.zip...\n",
      "Downloading https://github.com/ultralytics/assets/releases/download/v0.0.0/VisDrone2019-DET-train.zip to /root/autodl-tmp/VisDrone/VisDrone2019-DET-train.zip...\n",
      "Downloading https://github.com/ultralytics/assets/releases/download/v0.0.0/VisDrone2019-DET-test-dev.zip to /root/autodl-tmp/VisDrone/VisDrone2019-DET-test-dev.zip...\n",
      "Unzipping /root/autodl-tmp/VisDrone/VisDrone2019-DET-val.zip...\n",
      "Unzipping /root/autodl-tmp/VisDrone/VisDrone2019-DET-test-challenge.zip...\n",
      "Unzipping /root/autodl-tmp/VisDrone/VisDrone2019-DET-test-dev.zip...\n",
      "Unzipping /root/autodl-tmp/VisDrone/VisDrone2019-DET-train.zip...\n",
      "Converting /root/autodl-tmp/VisDrone/VisDrone2019-DET-train: 6471it [00:55, 115.56it/s]\n",
      "Converting /root/autodl-tmp/VisDrone/VisDrone2019-DET-val: 548it [00:05, 94.64it/s] \n",
      "Converting /root/autodl-tmp/VisDrone/VisDrone2019-DET-test-dev: 1610it [00:11, 144.76it/s]\n"
     ]
    }
   ],
   "source": [
    "import subprocess\n",
    "import os\n",
    "\n",
    "result = subprocess.run('bash -c \"source /etc/network_turbo && env | grep proxy\"', shell=True, capture_output=True, text=True)\n",
    "output = result.stdout\n",
    "for line in output.splitlines():\n",
    "    if '=' in line:\n",
    "        var, value = line.split('=', 1)\n",
    "        os.environ[var] = value  # 要从github下载东西，这个是autodl开代理的可以下快点。\n",
    "        \n",
    "from Dataset_process import *\n",
    "download_VisDrone()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "0a16150f-7fcb-41f2-8a35-2b7e1506ff52",
   "metadata": {},
   "outputs": [],
   "source": [
    "command = f\"\"\" \\\n",
    "rm -rf ../datasets/VisDrone && \\\n",
    "mkdir ../datasets/VisDrone && \\\n",
    "mkdir ../datasets/VisDrone/images && \\\n",
    "mkdir ../datasets/VisDrone/labels && \\\n",
    "mkdir ../datasets/VisDrone/images/VisDrone2019-DET-train/ && \\\n",
    "mkdir ../datasets/VisDrone/images/VisDrone2019-DET-val/ && \\\n",
    "mkdir ../datasets/VisDrone/images/VisDrone2019-DET-test-dev/ && \\\n",
    "mkdir ../datasets/VisDrone/labels/VisDrone2019-DET-train/ && \\\n",
    "mkdir ../datasets/VisDrone/labels/VisDrone2019-DET-val/ && \\\n",
    "mkdir ../datasets/VisDrone/labels/VisDrone2019-DET-test-dev/ && \\\n",
    "cp ../VisDrone/VisDrone2019-DET-train/images/* ../datasets/VisDrone/images/VisDrone2019-DET-train/ && \\\n",
    "cp ../VisDrone/VisDrone2019-DET-val/images/* ../datasets/VisDrone/images/VisDrone2019-DET-val/ && \\\n",
    "cp ../VisDrone/VisDrone2019-DET-test-dev/images/* ../datasets/VisDrone/images/VisDrone2019-DET-test-dev/ && \\\n",
    "cp ../VisDrone/VisDrone2019-DET-train/labels/* ../datasets/VisDrone/labels/VisDrone2019-DET-train/ && \\\n",
    "cp ../VisDrone/VisDrone2019-DET-val/labels/* ../datasets/VisDrone/labels/VisDrone2019-DET-val/ && \\\n",
    "cp ../VisDrone/VisDrone2019-DET-test-dev/labels/* ../datasets/VisDrone/labels/VisDrone2019-DET-test-dev/ \\\n",
    "\"\"\"\n",
    "!{command}\n",
    "#18：58"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "70ff5de7-1f6d-438d-bb99-cb87d9f25498",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
